Connection Classification

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.

suppressMessages(library(tidyverse))
suppressMessages(library(stringr))
suppressMessages(library(ISLR))
suppressMessages(library(caret))
suppressMessages(library(doMC))
registerDoMC(cores=4)

Add a new chunk by clicking the Insert Chunk button on the toolbar or by pressing Ctrl+Alt+I.

When you save the notebook, an HTML file containing the code and output will be saved alongside it (click the Preview button or press Ctrl+Shift+K to preview the HTML file).

Getting and proccesing the data

library(stringr)
myData = read.csv('/home/harpo/Dropbox/ongoing-work/git-repos/labeling-datasets/phd/datasets/data_all_result.txt', stringsAsFactors = F, sep = ' ')
#Create data backup
myData.bkup <- myData
#Create new column: length of model, and number of periodicity, duration and size characteristic in the model.
myData = myData %>% mutate(letter_count = nchar(State))
#Periodicity
myData = myData %>% mutate(strong_p = str_count(State,'[a-i]'))
myData = myData %>% mutate(weak_p = str_count(State,'[A-I]'))
myData = myData %>% mutate(weak_np = str_count(State,'[r-z]'))
myData = myData %>% mutate(strong_np = str_count(State,'[R-Z]'))
#Duration
myData = myData %>% mutate(duration_s = str_count(State,'(a|A|r|R|1|d|D|u|U|4|g|G|x|X|7)'))#c('a','A','r','R','1','d','D','u','U','4','g','G','x','X','7')))
myData = myData %>% mutate(duration_m = str_count(State,'(b|B|s|S|2|e|E|v|V|5|h|H|y|Y|8)'))#c('b','B','s','S','2','e','E','v','V','5','h','H','y','Y','8')))
myData = myData %>% mutate(duration_l = str_count(State,'(c|C|t|T|3|f|F|w|W|6|i|I|z|Z|9)'))#c('c','C','t','T','3','f','F','w','W','6','i','I','z','Z','9')))
#Size
myData = myData %>% mutate(size_s = str_count(State,'[a-c]') + str_count(State,'[A-C]') + str_count(State,'[r-t]') + str_count(State,'[R-T]') + str_count(State,'[1-3]'))
myData = myData %>% mutate(size_m = str_count(State,'[d-f]') + str_count(State,'[D-F]') + str_count(State,'[u-w]') + str_count(State,'[U-W]') + str_count(State,'[4-6]'))
myData = myData %>% mutate(size_l = str_count(State,'[g-i]') + str_count(State,'[G-I]') + str_count(State,'[x-z]') + str_count(State,'[X-Z]') + str_count(State,'[7-9]'))
#Remove from LabelName unnecessary characters (ej: V42, -17)
myData <- myData %>% mutate(LabelName = gsub('V[0-9]+-','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('-[0-9]+','',LabelName))
myData <- myData %>% mutate(LabelName = gsub('CC[0-9]+-','CC-',LabelName))
#Keep only connection with more than 3 symbols
myData <- myData %>% filter(letter_count > 3)
#Periodicity %
myData <- myData %>% mutate(strong_p = (strong_p / letter_count))
myData <- myData %>% mutate(weak_p = (weak_p / letter_count))
myData <- myData %>% mutate(strong_np = (strong_np / letter_count))
myData <- myData %>% mutate(weak_np = (weak_np / letter_count))
#Duration %
myData <- myData %>% mutate(duration_s = (duration_s / letter_count))
myData <- myData %>% mutate(duration_m = (duration_m / letter_count))
myData <- myData %>% mutate(duration_l = (duration_l / letter_count))
#Size %
myData <- myData %>% mutate(size_s = (size_s / letter_count))
myData <- myData %>% mutate(size_m = (size_m / letter_count))
myData <- myData %>% mutate(size_l = (size_l / letter_count))
#head(myData)
myData[1:20,]

#Making feature vectors
feature_vectors = myData[,c('strong_p','weak_p','weak_np','strong_np','duration_s','duration_m','duration_l','size_s','size_m','size_l','letter_count','Label','LabelName')]
names(feature_vectors) = c("sp","wp","wnp","snp","ds","dm","dl","ss","sm","sl","length","class","subclass")
feature_vectors$class = factor(feature_vectors$class)
feature_vectors$subclass = factor(feature_vectors$subclass)

Create training set and testset

Random Forest Classificator

  # Random Forest
rfFit <- train(class ~ .,
               data = training,
               metric="ROC",
               method = "rf",
               trControl = ctrl_fast)

rfFit
rfFit$finalModel

predsrfprobs=predict(rfFit,testing,type='prob')
predsrf=ifelse(predsrfprobs$Botnet >=0.9,'Botnet','Normal')
confusionMatrix(predsrf,testing$class)

library(ggplot2)
library(plotROC)
selectedIndices <- rfFit$pred$mtry == 2
ggplot(cbind(predsrfprobs,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

cbind(predsrfprobs,class=testing$class)

KNN

#Checking distibution in origanl data and partitioned data
prop.table(table(training$class)) * 100
prop.table(table(testing$class)) * 100
prop.table(table(feature_vectors$class)) * 100

trainX <- training[,names(training) != "class"]
preProcValues <- preProcess(x = trainX,method = c("center", "scale"))
preProcValues

knnFit <- train(class ~ ., data = training, method = "knn", trControl = ctrl_fast, preProcess = c("center","scale"), tuneLength = 20)

#Output of kNN fit
knnFit

#Plotting yields Number of Neighbours Vs accuracy (based on repeated cross validation)
plot(knnFit)

knnPredict <- predict(knnFit,newdata = testing )
#Get the confusion matrix to see accuracy value and other parameter values
confusionMatrix(knnPredict, testing$class )

mean(knnPredict == testing$class)

library(pROC)
knnPredict <- predict(knnFit,newdata = testing , type="prob")
knnROC <- roc(testing$class,knnPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
knnROC

ggplot(cbind(knnPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(knnROC, type="S", print.thres= 0.5)

Logistic Regression

logicRFit <- train(class ~ ., method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#logicRFit <- train(class ~ sp*wp*wnp*snp*ds*dm*dl*ss*sm*sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
#logicRFit <- train(class ~ sp+wp+wnp+snp+ds+dm+dl+ss+sm+sl, method='glm', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))

#summary(logicRFit)
#Output of Logistic Regression fit
logicRFit

logicRPredict <- predict(logicRFit, newdata = testing )

confusionMatrix(logicRPredict, testing$class)

logicRPredict <- predict(logicRFit, newdata = testing, type="prob")
logicROC <- roc(testing$class,logicRPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))

ggplot(cbind(logicRPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#logicROC

Naive Bayes

naiveBayesFit <- train(class ~ ., method='nb', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training)
naiveBayesFit

naiveBayesPredict <- predict(naiveBayesFit, newdata = testing)

confusionMatrix(naiveBayesPredict, testing$class)

naiveBayesPredict <- predict(naiveBayesFit, newdata = testing, type = 'prob')
naiveBayesROC <- roc(testing$class,naiveBayesPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
naiveBayesROC

ggplot(cbind(naiveBayesPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

#plot(naiveBayesROC, type="S", print.thres= 0.5)

Suport Vector Machine

svmFit <- train(class ~ ., method='svmLinear', trControl = ctrl_fast,preProcess=c('scale', 'center'), data=training, family=binomial(link='logit'))
svmFit

svmPredict <- predict(svmFit, newdata = testing)
confusionMatrix(svmPredict, testing$class)

svmPredict <- predict(svmFit, newdata = testing, type = "prob")

svmROC <- roc(testing$class,svmPredict[,"Botnet"], levels = c('Normal','Botnet'))#rev(testing$class))
svmROC

ggplot(cbind(svmPredict,class=testing$class), 
       aes(m = Botnet, d = factor(class, labels=c("Normal","Botnet"),levels = c("Normal", "Botnet")))) + 
    geom_roc(hjust = -0.4, vjust = 1.5,colour='orange') + 
  theme_bw()

Comparing Models

resamps <- resamples(list(rf = rfFit, lr = logicRFit, nv = naiveBayesFit, svm = svmFit))
summary(resamps)
bwplot(resamps)
diffs <- diff(resamps)
summary(diffs)
values=resamps$values
values
names(values)[2]<-"rfSens"

ggplot(values)+
  geom_boxplot(aes(y=rfSens,x=1))

Making probabilistic table.

# Botnet probabilistic table
botnet_prob_result = data.frame(testing$class, predsrfprobs$Botnet, knnPredict$Botnet, logicRPredict$Botnet, naiveBayesPredict$Botnet ,svmPredict$Botnet)
names(botnet_prob_result) = c('TrueClass','RamdomForest','KNN','LogisticRegression', 'NaiveBayes', 'SVM','subclass')

#load("./botnet_prob_results.Rda")
library(grid)
library(gridExtra)
botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),mean=mean(KNN),sd=sd(KNN)) %>% arrange(desc(n))

botnet_prob_result %>% group_by(subclass) %>% summarise(n=n(),mean=mean(NaiveBayes),sd=sd(NaiveBayes)) %>% arrange(desc(n))%>% top_n(10)

Selecting by sd

botnet_10_top<-botnet_prob_result %>% group_by(subclass) %>% summarise(n=n()) %>% arrange(desc(n))%>% top_n(10)

Selecting by n

botnet_10_top<-inner_join(botnet_10_top,botnet_prob_result,by="subclass")
knn_plot<-bwplot(subclass~KNN,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
rf_plot<-bwplot(subclass~RamdomForest,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
rl_plot<-bwplot(subclass~LogisticRegression,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
svm_plot<-bwplot(subclass~SVM,data=botnet_10_top,do.out = FALSE,scales=list(y=list(draw=FALSE)))
pl = list(knn_plot, rf_plot,rl_plot,svm_plot)
# do.call(grid.arrange, c(pl, nrow=1))
do.call(grid.arrange, c(lapply(pl, update), list(nrow=1)))

some heatmaps

difference heatmaps

Clustering and PCA

kmeans_mod<-kmeans(testing[,1:10],centers = 10)
testing_cluster<-cbind(testing,cluster=kmeans_mod$cluster)
pca<-prcomp(testing[,c(-11,-12)], center = TRUE, scale. = TRUE) 
pca_testing<-data.frame(pca$x,class=testing_cluster$class,
                              subclass=testing_cluster$subclass,
                               cluster=testing_cluster$cluster
                               )
g<-ggplot(pca_testing,aes(x=PC1,y=PC2))+
  geom_jitter(aes(color=as.factor(subclass),text=cluster,shape=class))+
  #geom_point(aes(shape=asignacion),size=3)+
  ylab("PC1")+xlab("PC2")+
  theme_classic()+
#scale_shape_manual(values=c(8,6))+
   guides(color=FALSE,alpha=FALSE)

Ignoring unknown aesthetics: text

ggplotly(g)

#Normal probabilistic table
normal_prob_result = data.frame(testing$class, predsrfprobs$Normal, knnPredict$Normal, logicRPredict$Normal, naiveBayesPredict$Normal ,svmPredict$Normal)
names(normal_prob_result) = c('True Class','Ramdom Forest','KNN','Logistic Regression', 'Naive Bayes', 'Suport VM')
normal_prob_result